{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de765f16-341b-4dc3-b59d-b16437c7e050",
   "metadata": {},
   "source": [
    "\n",
    "# RAG with structurizr sample codes downloaded from github\n",
    "\n",
    "Please refer to the content of dsl code below in this demo.\n",
    "\n",
    "[examples/dsl/big-bank-plc/model.dsl](examples/dsl/big-bank-plc/model.dsl)\n",
    "\n",
    "The code is clone from `https://github.com/structurizr/examples.git`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e25cdbdc-daee-4a9b-b363-9080dc216927",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: langchain\n",
      "Version: 0.1.9\n",
      "Summary: Building applications with LLMs through composability\n",
      "Home-page: https://github.com/langchain-ai/langchain\n",
      "Author: \n",
      "Author-email: \n",
      "License: MIT\n",
      "Location: /opt/conda/lib/python3.11/site-packages\n",
      "Requires: aiohttp, dataclasses-json, jsonpatch, langchain-community, langchain-core, langsmith, numpy, pydantic, PyYAML, requests, SQLAlchemy, tenacity\n",
      "Required-by: \n"
     ]
    }
   ],
   "source": [
    "!pip show langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e07b2c7d-62d7-457c-bbcb-298b29aa522c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Please define the following env variables in the .env file \n",
    "# \n",
    "# ollama_url = \"http://ollama:11434\"\n",
    "# model_name = \"mistral:instruct\"\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fa2048ea-dbc4-49d7-b855-8b546dce2edb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use the utility.py to get llm, embedding and output parser\n",
    "import utility\n",
    "embeddings = utility.get_embeddings()\n",
    "llm = utility.get_llm()\n",
    "output_parser = utility.get_output_parser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a11baec0-71af-46f8-bb39-bacc1c354c55",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:00<00:00, 1351.26it/s]\n"
     ]
    }
   ],
   "source": [
    "# Load the dsl file from local machine for embedding\n",
    "# https://python.langchain.com/docs/modules/data_connection/document_loaders/file_directory\n",
    "# https://python.langchain.com/docs/integrations/document_loaders/unstructured_file\n",
    "\n",
    "from langchain_community.document_loaders import DirectoryLoader, TextLoader\n",
    "\n",
    "# only load examples/dsl/big-bank-plc/model.dsl\n",
    "loader = DirectoryLoader('examples/dsl/big-bank-plc', glob=\"model.dsl\", show_progress=True, use_multithreading=True, loader_cls=TextLoader)\n",
    "\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "97c4e0ee-d1b7-4f11-914c-4b19d7812b06",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(page_content='workspace {\\n\\n    model {\\n        customer = person \"Personal Banking Customer\" \"A customer of the bank, with personal bank accounts.\" \"Customer\"\\n\\n        group \"Big Bank plc\" {\\n            supportStaff = person \"Customer Service Staff\" \"Customer service staff within the bank.\" \"Bank Staff\"\\n            backoffice = person \"Back Office Staff\" \"Administration and support staff within the bank.\" \"Bank Staff\"\\n\\n            mainframe = softwaresystem \"Mainframe Banking System\" \"Stores all of the core banking information about customers, accounts, transactions, etc.\" \"Existing System\"\\n            email = softwaresystem \"E-mail System\" \"The internal Microsoft Exchange e-mail system.\" \"Existing System\"\\n            atm = softwaresystem \"ATM\" \"Allows customers to withdraw cash.\" \"Existing System\"\\n\\n            internetBankingSystem = softwaresystem \"Internet Banking System\" \"Allows customers to view information about their bank accounts, and make payments.\"\\n        }\\n\\n        # relationships between people and software systems\\n        customer -> internetBankingSystem \"Views account balances, and makes payments using\"\\n        internetBankingSystem -> mainframe \"Gets account information from, and makes payments using\"\\n        internetBankingSystem -> email \"Sends e-mail using\"\\n        email -> customer \"Sends e-mails to\"\\n        customer -> supportStaff \"Asks questions to\" \"Telephone\"\\n        supportStaff -> mainframe \"Uses\"\\n        customer -> atm \"Withdraws cash using\"\\n        atm -> mainframe \"Uses\"\\n        backoffice -> mainframe \"Uses\"\\n    }\\n\\n}', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'})]\n"
     ]
    }
   ],
   "source": [
    "# print the loaded documents, the model.dsl content has been loaded as the page_content\n",
    "print(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "dea60994-7061-48d3-a1de-8617f447078e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Created a chunk of size 261, which is longer than the specified 200\n",
      "Created a chunk of size 386, which is longer than the specified 200\n",
      "Created a chunk of size 588, which is longer than the specified 200\n"
     ]
    }
   ],
   "source": [
    "# Split the document content into small chunks of texts for embedding\n",
    "\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "text_splitter = CharacterTextSplitter(chunk_size=200, chunk_overlap=50)\n",
    "texts = text_splitter.split_documents(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4070da74-d1c9-4158-8684-2c8c243993de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Document(page_content='workspace {\\n\\n    model {\\n        customer = person \"Personal Banking Customer\" \"A customer of the bank, with personal bank accounts.\" \"Customer\"', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'}), Document(page_content='group \"Big Bank plc\" {\\n            supportStaff = person \"Customer Service Staff\" \"Customer service staff within the bank.\" \"Bank Staff\"\\n            backoffice = person \"Back Office Staff\" \"Administration and support staff within the bank.\" \"Bank Staff\"', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'}), Document(page_content='mainframe = softwaresystem \"Mainframe Banking System\" \"Stores all of the core banking information about customers, accounts, transactions, etc.\" \"Existing System\"\\n            email = softwaresystem \"E-mail System\" \"The internal Microsoft Exchange e-mail system.\" \"Existing System\"\\n            atm = softwaresystem \"ATM\" \"Allows customers to withdraw cash.\" \"Existing System\"', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'}), Document(page_content='internetBankingSystem = softwaresystem \"Internet Banking System\" \"Allows customers to view information about their bank accounts, and make payments.\"\\n        }', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'}), Document(page_content='# relationships between people and software systems\\n        customer -> internetBankingSystem \"Views account balances, and makes payments using\"\\n        internetBankingSystem -> mainframe \"Gets account information from, and makes payments using\"\\n        internetBankingSystem -> email \"Sends e-mail using\"\\n        email -> customer \"Sends e-mails to\"\\n        customer -> supportStaff \"Asks questions to\" \"Telephone\"\\n        supportStaff -> mainframe \"Uses\"\\n        customer -> atm \"Withdraws cash using\"\\n        atm -> mainframe \"Uses\"\\n        backoffice -> mainframe \"Uses\"\\n    }', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'}), Document(page_content='}', metadata={'source': 'examples/dsl/big-bank-plc/model.dsl'})]\n"
     ]
    }
   ],
   "source": [
    "# print the chucks, the model.dsl content has been splitted into small documents with page_content of \n",
    "print(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "93f03020-3887-4aae-9ece-6bd088d51d49",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from langchain_community.vectorstores import DocArrayInMemorySearch\n",
    "\n",
    "vectorstore = DocArrayInMemorySearch.from_documents(texts, embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "02b3f72b-a7e1-48f9-8a08-e61399644775",
   "metadata": {},
   "outputs": [],
   "source": [
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "081c8e88-e45c-4b32-bc0e-206801565f83",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnableParallel, RunnablePassthrough\n",
    "\n",
    "retrieval = RunnableParallel(\n",
    "    {\"context\": retriever, \"input\": RunnablePassthrough()}\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dd527cf0-379d-4800-bff7-02b7cfdf41c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "\n",
    "prompt = ChatPromptTemplate.from_messages([\n",
    "    (\"system\", \"You are a software architect with knowledage in structurizr helping software architecture modeling and documenation in the following context: {context}\"),\n",
    "    (\"user\", \"{input}\")\n",
    "])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fd5a6f70-6f06-4360-b35f-8a5509c36bc8",
   "metadata": {},
   "outputs": [],
   "source": [
    "chain = retrieval | prompt | llm | output_parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "336770a5-a375-4199-9759-bf1753ff774a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the provided context in the given documents, the \"Internet Banking System\" is a software system within the Big Bank plc organization. It allows customers to view information about their bank accounts and make payments online. This system interacts with other systems such as the Customer system and the Core Banking Information system (Mainframe), which store the necessary data for internet banking transactions.\n"
     ]
    }
   ],
   "source": [
    "result = chain.invoke(\"\"\"What is the \"Internet Banking System\" \"\"\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b2ec70bc-9810-4af9-af64-0a507958cc83",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " The \"E-mail System\" referred to in this context is an existing software system named \"email\". It's described as the internal Microsoft Exchange e-mail system. This system is not further detailed, but it can be inferred that it plays a role in handling and managing email communications within the organization of the Big Bank plc."
     ]
    }
   ],
   "source": [
    "# print(chain.invoke(\"\"\"What is the \"E-mail System\" \"\"\"))\n",
    "for chunk in chain.stream(\"\"\"What is the \"E-mail System\" \"\"\"):\n",
    "    print(chunk, end=\"\", flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "23a2835a-bd0e-42ad-af42-8b05afb09906",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the provided context from the given documents in Structurizr DSL language, there are two defined groups of bank staff: \"Customer Service Staff\" and \"Back Office Staff\". These staff members are represented as persons under their respective groups within the \"Big Bank plc\" entity. Therefore, the bank staff include customer service staff and back office staff.\n"
     ]
    }
   ],
   "source": [
    "# sample to run a batch of queries\n",
    "for result in chain.batch([\"\"\"Who are the bank staff \"\"\"]):\n",
    "    print(result,flush=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "09df3b1e-40ee-4f2e-926b-9c722395328a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Based on the user requirements, here is the Structurizr DSL code:\n",
      "\n",
      "```dsl\n",
      "workspace {\n",
      "    name \"Big Bank plc\"\n",
      "    description \"A model of the Big Bank plc software architecture\"\n",
      "\n",
      "    model {\n",
      "        customer = person \"Customer\"\n",
      "        internetBankingSystem = softwareSystem \"Internet Banking System\"\n",
      "        mainframe = softwareSystem \"Mainframe Banking System\"\n",
      "        email = softwareSystem \"E-mail System\"\n",
      "        atm = softwareSystem \"ATM\"\n",
      "        bankStaff = person \"Bank Staff\"\n",
      "\n",
      "        customer -> internetBankingSystem \"Views account information, makes payments\"\n",
      "        internetBankingSystem -> mainframe \"Gets customer information\"\n",
      "        customer -> atm \"Withdraws cash\"\n",
      "        atm -> mainframe \"Uses mainframe for transactions\"\n",
      "        bankStaff -> mainframe \"Searches for customer information\"\n",
      "\n",
      "    }\n",
      "\n",
      "    views {\n",
      "        systemLandscape softwareSystem {\n",
      "            include *\n",
      "            autolayout lr\n",
      "        }\n",
      "    }\n",
      "}\n",
      "```\n",
      "\n",
      "This code defines a workspace named \"Big Bank plc\", which includes five persons and five software systems. The relationships between these elements are defined using the \"->\" operator. For example, \"customer -> internetBankingSystem\" represents the relationship where customers use the Internet Banking System to view account information and make payments. Similarly, \"internetBankingSystem -> mainframe\" represents the relationship where the Internet Banking System gets customer information from the Mainframe Banking System. The code adheres to the specified format and does not embed relationships within software systems as required."
     ]
    }
   ],
   "source": [
    "# requirements = \"\"\"The \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "# the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "# \"\"\"\n",
    "\n",
    "instruct = \"\"\" Please generate the structurizr DSL codes according to user requirements: \n",
    "1. the \"Big Bank plc\" wants to create a new application for its bank staff and customers, \n",
    "2. the customers will use \"ATM\" system to withdraw cash, the bank staff will use mainframe system to search customer information.\n",
    "\n",
    "The Structurizr DSL code should follow the following formats: \n",
    "1. it must start with \"workspace {\" as the first line\n",
    "2. it must end with \"}\" as the last line\n",
    "3. it must have a section called \"model\"\n",
    "4. it must follow pattern \"systemA -> systemB\" to define relationship\n",
    "5. it must not embeb \"relationship\" within \"softwareSystem\"\n",
    "6. it must use this pattern to define softwareSystem, pattern: id = softwareSystem \"Software System Name\" \n",
    "7. it must adhere to the template below:\n",
    "\"workspace {\n",
    "    name \"placehoder for application name\"\n",
    "    descritption \"placehoder for application descritption\"\n",
    "\n",
    "    model {\n",
    "        //define the persons here\n",
    "        user = person \"User\"\n",
    "        \n",
    "        //define the software systems here, it can have multiple systems\n",
    "        softwareSystem = softwareSystem \"Software System\" \n",
    "        \n",
    "        //define the relationship here\n",
    "        user -> webapp \"Uses\"\n",
    "        webapp -> database \"Reads from and writes to\"\n",
    "        \n",
    "    }\n",
    "\n",
    "    //do not change this section\n",
    "    views {\n",
    "        systemLandscape softwareSystem {\n",
    "            include *\n",
    "            autolayout lr\n",
    "        }\n",
    "    }\n",
    "\n",
    "}\"\n",
    "\"\"\"\n",
    "\n",
    "for chunk in chain.stream(instruct):\n",
    "    print(chunk, end=\"\", flush=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8593a8f-fd78-413c-95cd-07450af7cf4a",
   "metadata": {},
   "source": [
    "\n",
    "## Notes\n",
    "\n",
    "Open https://structurizr.com/dsl and put the above dsl codes into the online editor to see the architecture diagram.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "40b96bb3-766a-4c6d-bd51-da60d283f77d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
